import os,sys
import re
import string
import datetime
from sets import Set

def fatal():
	sys.exit()

#a comparing function used for sort function to compare two WordData
#structures.
def compare(E1, E2):
       if cmp(E1.count, E2.count) != 0:
	    return (-1) * cmp(E1.count, E2.count)
       else: 
       	    return cmp(E1.word, E2.word)

#The main TextConcord class
class TextConcord:

	#the constructor of this class
	def __init__(self):
			#a dictionary recording the mapping from a word
			#to a WordData structure
		self.concord = {}

	#the actually procedure of calculate the concordance.
	def doConcordance(self, dir):
		#get all the files in the directory
		files = os.listdir(dir)
		for file in files:
			#skip files starting with .
			if(file[0] == '.'):
				continue
			path = dir + file
			#parse files one by one
			self.parse(path)
		self.output()
	
	#output the concordance of words according the their frequency
	#and their alphabetical order
	def output(self):
		#a set recording WordData structures
		data_values = set()
		for value in self.concord.values():
			data_values.add(value)

		#change the set to the type of list
		value_list = list(data_values);
		#sort the list according to the compare function we
		#provide
		value_list = sorted(value_list, compare)
		#construct the name of output file name
		outputFileName = "concord." + \
		str(datetime.datetime.now().microsecond)+".out"
		#write words to the outputfile
		bw = open(outputFileName,'a')
		for value in value_list:
			bw.write(value.word)
			bw.write("\n");
		bw.close()
	
	#parse a file into a collection of words
	def parse(self, file):
#		print "%s" %(file)
		br = open(file)
		for line in br:
			#words in one line are split according to the 
			#regular expression below
			words = re.split('[\\s,;.\"()\\[\\]:?!<>{}*]',line)
			for word in words:
				#for each word, add it to the dictionary
				self.addWord(word, file)

	#add a word to the dictionary
	def addWord(self, word, fileName):
		if cmp(word, "") == 0:
			return
		#change all words to lowercases and compare
		#if this word is a key of the dictionary, 
		#just add one in frequency and store the filename
		#in the filename set
		if word.lower() in self.concord:
			data = self.concord[word.lower()]
			data.count += 1
			data.fileNames.add(fileName)
		#if not, create a new WordData structure and add
		#the map to the dictionary
		else:
			data = WordData()
			data.word = word.lower()
			data.fileNames.add(fileName)
			self.concord[word.lower()] = data

#the definition of class WordData
class WordData:

	def __init__(self):
		self.word = ""
		self.count = 1
		self.fileNames = set()

#main function
if len(sys.argv)>1:
	dir = sys.argv[1]
else:
	fatal()

if os.path.isfile(sys.argv[1]) :
	print "Error: %s is not a directory" %(sys.argv[1])
	fatal()

tc = TextConcord ()
tc.doConcordance(dir)
print "Did concordance"
